﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using HtmlAgilityPack;
using Utilities.StringLib;

namespace DTO
{   
    public class Webpage
    {
        #region Attributes
        private static string localRepositoryPath;
        int id;
        string path;
        string url;
        string originalUrl;
        string title;
        string plainText;
        Dictionary<string, Link> outgoingLinks;
        List<string> incomingLinkTexts;
        List<string> metaKeywords;
        List<Webtag> textTagList;
        Webtag webtagTree;
        #endregion

        #region Properties
        public Webtag WebtagTree
        {
            get
            {
                return webtagTree;
            }
            set
            {
                webtagTree = value;
            }
        }
        
        public static string LocalRepositoryPath
        {
            get
            {
                return Webpage.localRepositoryPath;
            }
            set
            {
                Webpage.localRepositoryPath = value;
            }
        }

        /// <summary>
        /// The original Url of this webpage.
        /// </summary>
        public string OriginalUrl
        {
            get
            {
                return originalUrl;
            }
            set
            {
                originalUrl = value;
            }
        }

        /// <summary>
        /// List of all lead tags in this webpage
        /// </summary>
        public List<Webtag> TextTagList
        {
            get
            {
                return textTagList;
            }
            set
            {
                textTagList = value;
            }
        }

        /// <value>
        /// ID of this webpage.
        /// </value>
        public int ID
        {
            get
            {
                return id;
            }
            set
            {
                id = value;
            }
        }

        /// <value>
        /// Relative physical path of this webpage.
        /// </value>
        public string Path
        {
            get
            {
                return path;
            }
            set
            {
                path = value;
            }
        }

        /// <value>
        /// Plain text of this webpage.
        /// </value>
        public string PlainText
        {
            get
            {
                return plainText;
            }
            set
            {
                plainText = value;
            }
        }

        /// <value>
        /// Title of this webpage.
        /// </value>
        public string Title
        {
            get
            {
                return title;
            }
            set
            {
                title = value;
            }
        }

        /// <value>
        /// List of meta keywords of this webpage.
        /// </value>
        public List<string> MetaKeywords
        {
            get
            {
                return metaKeywords;
            }
            set
            {
                metaKeywords = value;
            }
        }

        /// <value>
        /// Dictionary of relative outgoing links (xPath,URL) of this webpage.
        /// </value>
        public Dictionary<string, Link> OutgoingLinks
        {
            get
            {
                return outgoingLinks;
            }
            set
            {
                outgoingLinks = value;
            }
        }

        /// <value>
        /// List of all incoming link texts to this webpage.
        /// </value>
        public List<string> IncomingLinkTexts
        {
            get
            {
                return incomingLinkTexts;
            }
            set
            {
                incomingLinkTexts = value;
            }
        }

        /// <value>
        /// Relative URL of this webpage.
        /// </value>
        public string URL
        {
            get
            {
                return url;
            }
            set
            {
                url = value;
            }
        }

        #endregion

        #region Constructor
        public Webpage()
        {
            this.textTagList = new List<Webtag>();
            this.metaKeywords = new List<string>();
            this.outgoingLinks = new Dictionary<string,Link>();
            this.incomingLinkTexts = new List<string>();
        }
        #endregion

        #region Methods

        /// <summary>
        /// This methods convert a html webpage from a file into a webpage object
        /// </summary>
        /// <param name="htmlPath">Path of the html file on local repository</param>
        /// <returns>A webpage object</returns>
        public static Webpage HTMLWebpage2MyWebPage(string htmlPath)
        {
            // Load the content of the webpage into string
            StreamReader sr = new StreamReader(htmlPath);
            string htmlStr = sr.ReadToEnd();
            // Strip all multimedia and irrelevant tags
            //htmlStr = Webpage.StripMultimediaAndIrrelevantTag(htmlStr);
            // Load the html file using HTML Agility Pack
            HtmlDocument htmlDoc = new HtmlDocument();
            htmlDoc.LoadHtml(htmlStr);

            sr.Close();

            // Create a new webpage object
            Webpage curWebpage = new Webpage();
            curWebpage.MetaKeywords = Webpage.GetMetaKeywords(htmlDoc);
            curWebpage.Title = Webpage.GetTitle(htmlDoc);
            //curWebpage.OriginalUrl = Webpage.GetOriginalUrl(htmlDoc);

            GetLeafAndLink(curWebpage.outgoingLinks, curWebpage.textTagList, htmlDoc.DocumentNode.Descendants("body").First());
            curWebpage.textTagList = MergeTextTag(curWebpage.textTagList);

            return curWebpage;
        }

        private static List<Webtag> MergeTextTag(List<Webtag> curList)
        {
            //curList.Sort(CompareWebTagByXpath);

            List<Webtag> newTagList = new List<Webtag>();
            for (int i = 0; i < curList.Count; i++)
            {
                int j = i + 1;
                if (j < curList.Count)
                {
                    while (j < curList.Count && curList[j].XPath.CompareTo(curList[i].XPath) == 0)
                    {
                        j++;
                    }

                    if (j - 1 > i)
                    {
                        string tempText = string.Empty;
                        for (int m = i; m < j; m++)
                        {
                            tempText += " " + curList[m].Text;
                        }

                        Webtag tempWt = new Webtag();
                        tempWt.Name = curList[i].Name;
                        tempWt.XPath = curList[i].XPath;
                        tempWt.Text = tempText;

                        newTagList.Add(tempWt);
                        i = j - 1;
                    }
                    else
                    {
                        newTagList.Add(curList[i]);
                    }
                }
            }

            return newTagList;
        }

        private static int CompareWebTagByXpath(Webtag wtag1, Webtag wtag2)
        {
            int temp = wtag1.XPath.CompareTo(wtag2.XPath);
            return temp;
        }

        /// <summary>
        /// This method gets out of all multimedia and irrelevant tags of this webpage.
        /// </summary>
        /// <param name="html">The string that contains html document</param>
        /// <returns>Stripted html string</returns>
        private static string StripMultimediaAndIrrelevantTag(string html)
        {
            string regexPattern = @"</?(?i:script|style|img|center|hr|applet|object)(.|\n)*?>";
            return System.Text.RegularExpressions.Regex.Replace(html, regexPattern, "");
        }

        /// <summary>
        /// This method iterates through the webpage's tag tree to build a list of leaf tag.
        /// </summary>
        /// <param name="leafTagList">List of leaf tags</param>
        /// <param name="curNode">Current node</param>
        private static void GetLeafAndLink(Dictionary<string, Link> linkList, List<Webtag> textTagList, HtmlNode curNode)
        {
            if (curNode.NodeType == HtmlNodeType.Element
                && IsIrrelevant(curNode.Name) == true)
            {
                return;
            }
            else if (curNode.NodeType == HtmlNodeType.Element
                && curNode.Name.CompareTo("a") == 0)
            {
                string href = string.Empty;
                bool isExist = TestDesireableLink(curNode, out href);
                if (isExist == false)
                {
                    if (ProcessString.IsSpecialEmptyStr(curNode.InnerText) == false)
                    {
                        Webtag wtag = new Webtag();
                        wtag.Name = curNode.ParentNode.Name;
                        wtag.XPath = curNode.ParentNode.XPath;
                        wtag.Text = NormalizeString(curNode.InnerText);
                        textTagList.Add(wtag);
                    }
                }
                else
                {
                    Link curLink = new Link(href, NormalizeString(curNode.InnerText));
                    Webtag wtag = new Webtag();
                    wtag.Name = curNode.Name;
                    wtag.XPath = curNode.XPath;
                    wtag.Text = NormalizeString(curNode.InnerText);
                    textTagList.Add(wtag);
                    linkList.Add(wtag.XPath, curLink);
                }

                return;
            }

            foreach (HtmlNode i in curNode.ChildNodes)
            {
                if (i.NodeType == HtmlNodeType.Text)
                {
                    if (ProcessString.IsSpecialEmptyStr(i.InnerText) == false)
                    {
                        Webtag wtag = new Webtag();
                        wtag.Name = curNode.Name;
                        wtag.XPath = curNode.XPath;
                        wtag.Text = NormalizeString(i.InnerText);
                        textTagList.Add(wtag);
                    }
                }
                else if (IsIrrelevant(i.Name) == false)
                {
                    GetLeafAndLink(linkList, textTagList, i);
                }
            }
        }

        /// <summary>
        /// This method checks current link in this node if it links to an exist page of not.
        /// </summary>
        /// <param name="anchorNode">Current node that contains a link to other page.</param>
        /// <param name="href">The href in this node</param>
        /// <returns>True if this link is alive, else if not</returns>
        private static bool TestDesireableLink(HtmlNode anchorNode, out string href)
        {
            href = string.Empty;
            HtmlAttribute att = anchorNode.Attributes["href"];
            if (att == null)
            {
                return false;
            }

            href = att.Value;
            if (href.StartsWith("javascript", StringComparison.InvariantCultureIgnoreCase))
            {
                return false;
            }

            //char[] unwantedChars = new char[] { '"', (char)8220, (char)8221 }; //Chars which are removed: open quote, close quote

            //href = ProcessString.RemoveUnwantedChars(href, unwantedChars);
            //href = ProcessString.ReplaceUnwantedChars(href, "-", (char)65533); // 65533: &nbsp
            //href = ProcessString.RemoveDuplicatedChars(href, '-');

            if (File.Exists(localRepositoryPath + @"\" + href))
            {
                return true;
            }
            else
            {
                return false;
            }
        }

        private static bool IsIrrelevant(string tagName)
        {
            string[] irrelevantTags = Utilities.MyTagEnum.IrrelevantTag();
            for (int i = 0; i < irrelevantTags.Length; i++)
            {
                if (tagName.CompareTo(irrelevantTags[i]) == 0)
                {
                    return true;
                }
            }

            return false;
        }

        private static void GetTags(HtmlNode curNode, Webtag curWebtag)
        {
            foreach (HtmlNode i in curNode.ChildNodes)
            {
                if (i.NodeType == HtmlNodeType.Element)
                {
                    Webtag tempTag = new Webtag();
                    tempTag.Name = i.Name;
                    tempTag.XPath = i.XPath;
                    if (i.Name.Equals("a"))
                        tempTag.OriginalURL = i.Attributes["tppabs"].Value;
                    if (i.ChildNodes.Count == 1 && i.ChildNodes[0].ChildNodes.Count == 0)
                    {
                        if (i.InnerText.Contains('<') == false && i.InnerText.Contains('>') == false)
                            tempTag.Text = NormalizeString(i.InnerText);
                    }
                    curWebtag.Children.Add(tempTag);
                    GetTags(i, tempTag);
                }
            }
        }

        /// <summary>
        /// This method gets all outgoing links and their text from this webpage.
        /// </summary>
        /// <param name="htmlDoc">The html document needs to extract meta info.</param>
        private static List<Link> GetOutgoingLinks(HtmlDocument htmlDoc)
        {
            List<Link> outgoingLinks = new List<Link>();            
            try
            {
                foreach (HtmlNode link in htmlDoc.DocumentNode.SelectNodes(@"//a[@href]"))
                {                    
                    HtmlAttribute att = link.Attributes["href"];
                    if (att == null)
                    {
                        continue;
                    }
                    string href = att.Value;

                    if (href.StartsWith("javascript", StringComparison.InvariantCultureIgnoreCase))
                    {
                        continue;
                    }
                    outgoingLinks.Add(new Link(href, NormalizeString(link.InnerText)));
                }
                return outgoingLinks;
            }
            catch (Exception)
            {
                return outgoingLinks;
            }
        }

        /// <summary>
        /// This method converts the html content into plain text
        /// </summary>
        /// <param name="htmlStr">The html string needs to be converted into plain text.</param>
        /// <returns>Plain text format of this webpage</returns>
        private static string GetPlaintext(string htmlStr)
        {
            return NormalizeString(System.Text.RegularExpressions.Regex.Replace(htmlStr, @"<(.|\n)*?>", string.Empty));
        }

        /// <summary>
        /// This method gets all keywords from Meta tag of this webpage
        /// </summary>
        /// <param name="htmlDoc">The html document needs to be extracted meta info.</param>
        /// <returns>List of meta keywords of this webpage</returns>
        private static List<string> GetMetaKeywords(HtmlDocument htmlDoc)
        {
            HashSet<string> keywordHS = new HashSet<string>();
            string result = string.Empty;
            try
            {
                // Extract all meta node with attribute name = keywords
                foreach (HtmlNode curNode in htmlDoc.DocumentNode.Descendants("meta"))
                {
                    if ((curNode.Attributes["name"] != null) && (curNode.Attributes["name"].Value == "keywords"))
                    {
                        result += "," + curNode.Attributes["content"].Value.Trim();
                    }
                }

                string[] keywordList = result.Split(new char[] {','}, StringSplitOptions.RemoveEmptyEntries);
                
                foreach (string i in keywordList)
                {
                    keywordHS.Add(NormalizeString(i.Replace("\r\n", " ").Trim()));
                }

                // Split the result string into List<string>
                return Utilities.RefineKeyword.RefineKeywordList.Refine(keywordHS.ToList());
            }
            catch (Exception)
            {
                return keywordHS.ToList();
            }
        }

        /// <summary>
        /// This method gets the title of this webpage
        /// </summary>
        /// <param name="htmlDoc">The html document needs to be extracted title info.</param>
        /// <returns>Title of this webpage</returns>
        private static string GetTitle(HtmlDocument htmlDoc)
        {
            string result = string.Empty;
            try
            {
                return NormalizeString(htmlDoc.DocumentNode.Descendants("title").First().InnerText);
            }
            catch (Exception)
            {
                return string.Empty;
            }
        }

        /// <summary>
        /// This method gets the original Url of the current web page.
        /// </summary>
        /// <param name="htmlDoc">The html document needs to be extracted title info.</param>
        /// <returns>The original Url</returns>
        public static string GetOriginalUrl(HtmlDocument htmlDoc)
        {
            string url = string.Empty;

            foreach (HtmlNode i in htmlDoc.DocumentNode.Descendants("#comment"))
            {
                if (i.NodeType == HtmlNodeType.Comment)
                {
                    if (i.InnerText.Contains("url=") == true)
                    {
                        url = i.InnerText;
                        break;
                    }
                }
            }
            if (string.IsNullOrEmpty(url) == true)
                return string.Empty;

            int startIndex = url.IndexOf(')');
            return url.Substring(startIndex + 1, url.IndexOf(' ', startIndex + 1) - startIndex - 1);
        }

        /// <summary>
        /// Normalize a string
        /// </summary>
        /// <param name="input">The string needs to be normalized.</param>
        /// <returns>The normalized string</returns>
        private static string NormalizeString(string input)
        {
            char[] unwantedChars = new char[] { '"', (char)8220, (char)8221, '\t' }; //Chars which are removed: open quote, close quote
            string result = string.Empty;

            result = ProcessString.RenderString(input);
            result = ProcessString.RemoveUnwantedChars(result, unwantedChars);
            result = ProcessString.ReplaceUnwantedChars(result, "-", (char)65533); // 65533: &nbsp
            result = ProcessString.RemoveDuplicatedChars(result, '-', ' ');
            result = ProcessString.RemoveDuplicatedStrings(result, " -", "\r\n");
            result = ProcessString.RemoveEmptyRow(result);

            return result;
        }

        private static bool IsMetaKeywordsExist(HtmlDocument htmlDoc)
        {   
            return Webpage.GetMetaKeywords(htmlDoc).Count != 0;
        }

        private static bool IsOriginalUrlSuitable(HtmlDocument htmlDoc)
        {
            string originalURL = Webpage.GetOriginalUrl(htmlDoc);
            if (string.IsNullOrEmpty(originalURL))
                return false;
            // redirected page
            if (originalURL.IndexOf("http:") != originalURL.LastIndexOf("http:"))
                return false;
            return true;
        }

        public static bool IsDesireableWebpage(string htmlPath)
        {
            // Load the content of the webpage into string
            StreamReader sr = new StreamReader(htmlPath);
            string htmlStr = sr.ReadToEnd();
            HtmlDocument htmlDoc = new HtmlDocument();
            htmlDoc.LoadHtml(htmlStr);
            sr.Close();
            return IsMetaKeywordsExist(htmlDoc) & IsOriginalUrlSuitable(htmlDoc) ;
        }

        public static Dictionary<int, Webpage> WebpagesToDict(List<Webpage> allWebpages)
        {
            Dictionary<int, Webpage> allWebpagesDict = new Dictionary<int, Webpage>();

            foreach (Webpage curWebpage in allWebpages)
            {
                allWebpagesDict.Add(curWebpage.ID, curWebpage);
            }

            return allWebpagesDict;
        }

        #endregion
    }
}
